#!/usr/bin/env python 
# coding:utf-8
# @Time :10/8/18 17:48
import hashlib
import urllib
import ctypes
import urlparse

from tld import get_tld


def unicode2str(input):
    if isinstance(input, dict):
        return {unicode2str(key): unicode2str(value) for key, value in input.iteritems()}
    elif isinstance(input, list):
        return [unicode2str(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('utf-8')
    else:
        return input


def url_encode(url):
    if isinstance(url, unicode):
        url = url.encode('utf-8')
    url = url.replace('\t', '').replace('\n', '').replace('\r', '')
    url = urllib.quote(url, '!:?=/&%')
    return url


def get_url_info(base_url):
    url_split = urlparse.urlsplit(base_url)
    url_info = {}
    url_info['site'] = url_split.netloc
    url_info['site'] = url_info['site'].split(':')[0]
    url_info['site_id'] = get_md5_i64(url_info['site'])
    url_info['path'] = url_split.path
    url_info['query'] = url_split.query
    url_info['fragment'] = url_split.fragment
    try:
        url_info['domain'] = get_tld(base_url)
    except Exception as e:
        url_info['domain'] = url_info['site']
    if url_info.get('domain'):
        url_info['domain_id'] = get_md5_i64(url_info['domain'])
    else:
        url_info['domain_id'] = None

    url_info['url'] = base_url
    url_info['url_id'] = get_md5_i64(base_url)
    return url_info


def get_md5_i64(obj):
    m = hashlib.md5()
    m.update(obj)
    return ctypes.c_int64(int(m.hexdigest()[8:-8], 16)).value
